BLM data analysis

Installing required packages

library(rtweet)

package 㤼㸱rtweet㤼㸲 was built under R version 4.0.4

library(dplyr)
library(tidyr)
library(tidytext)

package 㤼㸱tidytext㤼㸲 was built under R version 4.0.4

library(ggplot.multistats)

package 㤼㸱ggplot.multistats㤼㸲 was built under R version 4.0.4

library(ggplot2)
library(forestmangr)

package 㤼㸱forestmangr㤼㸲 was built under R version 4.0.4
Attaching package: 㤼㸱forestmangr㤼㸲

The following object is masked from 㤼㸱package:data.table㤼㸲:

    :=

library(syuzhet)

package 㤼㸱syuzhet㤼㸲 was built under R version 4.0.4
Attaching package: 㤼㸱syuzhet㤼㸲

The following object is masked from 㤼㸱package:rtweet㤼㸲:

    get_tokens

library(tidyverse)

Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
-- Attaching packages ----------------------------- tidyverse 1.3.0 --
<U+221A> tibble  3.0.6     <U+221A> stringr 1.4.0
<U+221A> readr   1.4.0     <U+221A> forcats 0.5.1
<U+221A> purrr   0.3.4     
-- Conflicts -------------------------------- tidyverse_conflicts() --
x dplyr::between()   masks data.table::between()
x dplyr::filter()    masks stats::filter()
x dplyr::first()     masks data.table::first()
x purrr::flatten()   masks rtweet::flatten()
x dplyr::lag()       masks stats::lag()
x dplyr::last()      masks data.table::last()
x purrr::transpose() masks data.table::transpose()

library(readxl) # read excel
library(tibble) # tobble dataframe
library(dplyr) # piping
library(stringr) # character manipulation
library(tidytext)
library(tokenizers)

package 㤼㸱tokenizers㤼㸲 was built under R version 4.0.4

library(stopwords)

package 㤼㸱stopwords㤼㸲 was built under R version 4.0.4

library(tidyverse)
library(lobstr)

package 㤼㸱lobstr㤼㸲 was built under R version 4.0.4

Loading the data

blm2020<-read.csv("https://www.dropbox.com/s/7ywlwmynitdb2rr/BLM2020_Dataset.csv?dl=1", sep = '\t', na.strings = c("", "NA"))

blm2021<-read.csv("https://www.dropbox.com/s/glpv88v6hb0u4j1/BLM2021_Dataset.csv?dl=1", sep = '\t', na.strings = c("", "NA"))

Show the ratio of replies, retweets and likes

#Retweets
blm2020_retweets<-blm2020[blm2020$retweet > 1, ]
blm2020_retweets

blm2021_retweets<-blm2021[blm2021$retweet > 1, ]
blm2021_retweets

NA

#Likes
blm2020_likes<-blm2020[blm2020$like >1, ]
blm2020_likes

blm2021_likes<-blm2021[blm2021$like >1, ]
blm2021_likes

NA

#Replies
blm2020_replies<-blm2020[blm2020$reply >1, ]
blm2020_replies

blm2021_replies<-blm2021[blm2021$reply >1, ]
blm2021_replies

NA

Creating a data frame

data2020<-data.frame(category=c("Retweets", "Likes", "Replies"), count=c(2, 5, 3))
data2020

data2021<-data.frame(category=c("Retweets", "Likes", "Replies"), count=c(707, 1738, 574))
data2021

Visualizing the data 2020

#Adding columns
data2020$fraction = data2020$count / sum(data2020$count)
data2020$percentage = data2020$count / sum(data2020$count) * 100
data2020$ymax = cumsum(data2020$fraction)
data2020$ymin = c(0, head(data2020$ymax, n=-1))

#Rounding up

data2020 <- round_df(data2020, 2)

# Specify what the legend should say
Type_of_Tweet <- paste(data2020$category, data2020$percentage, "%")

ggplot(data2020, aes(ymax=ymax, ymin=ymin, xmax=4, xmin=3, fill=Type_of_Tweet)) +
  geom_rect() +
  coord_polar(theta="y") + 
  xlim(c(2, 4)) +
  theme_void() +
  theme(legend.position = "right")

Visualizing the data 2021

#Adding columns
data2021$fraction = data2021$count / sum(data2021$count)
data2021$percentage = data2021$count / sum(data2021$count) * 100
data2021$ymax = cumsum(data2021$fraction)
data2021$ymin = c(0, head(data2021$ymax, n=-1))

#Rounding up

data2021 <- round_df(data2021, 2)

# Specify what the legend should say
Type_of_Tweet <- paste(data2020$category, data2021$percentage, "%")

ggplot(data2021, aes(ymax=ymax, ymin=ymin, xmax=4, xmin=3, fill=Type_of_Tweet)) +
  geom_rect() +
  coord_polar(theta="y") + 
  xlim(c(2, 4)) +
  theme_void() +
  theme(legend.position = "right")

Most frequent words in the 2020 tweets

Lets see which words are more frequently used in the posts of 2020

blm2020$text <- gsub("@\\S*", "", blm2020$text)
blm2020$text <- gsub("amp", "", blm2020$text) 
blm2020$text <- gsub("[\r\n]", "", blm2020$text)
blm2020$text <- gsub("[[:punct:]]", "", blm2020$text)

#removing stop words from the text - so the most frequent words are not or at and

tweets <- blm2020 %>%
  select(text) %>%
  unnest_tokens(word, text)
tweets <- tweets %>%
  anti_join(get_stopwords(language = "nl", source = "snowball"))

Joining, by = "word"

#making a histogram
tweets %>% # gives you a bar chart of the most frequent words found in the tweets
  count(word, sort = TRUE) %>%
  top_n(15) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
  labs(y = "Count",
       x = "Unique words",
       title = "Most frequent words found in the tweets of #BLM",
       subtitle = "Stop words removed from the list")

Selecting by n

Most frequent words in the 2021 tweets Lets now see which words are more frequently used in the posts of 2021

blm2021$text <- gsub("@\\S*", "", blm2021$text)
blm2021$text <- gsub("amp", "", blm2021$text) 
blm2021$text <- gsub("[\r\n]", "", blm2021$text)
blm2021$text <- gsub("[[:punct:]]", "", blm2021$text)

#removing stop words from the text - so the most frequent words are not or at and

tweets1 <- blm2021 %>%
  select(text) %>%
  unnest_tokens(word, text)
tweets1 <- tweets1 %>%
  anti_join(get_stopwords(language = "nl", source = "snowball"))

Joining, by = "word"

#making a histogram
tweets1 %>% # gives you a bar chart of the most frequent words found in the tweets
  count(word, sort = TRUE) %>%
  top_n(15) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
  labs(y = "Count",
       x = "Unique words",
       title = "Most frequent words found in the tweets of #BLM 2021",
       subtitle = "Stop words removed from the list")

Selecting by n

Sentiment analysis for positive or negative tweets 2020

# Converting tweets to ASCII to trackle strange characters
tweets_sentiment <- iconv(tweets, from="UTF-8", to="ASCII", sub="")

# removing retweets, in case needed 
tweets_sentiment <-gsub("(RT|via)((?:\\b\\w*@\\w+)+)","",tweets_sentiment)

# removing mentions, in case needed
tweets_sentiment <-gsub("@\\w+","",tweets_sentiment)
ew_sentiment<-get_nrc_sentiment((tweets_sentiment))
sentimentscores<-data.frame(colSums(ew_sentiment[,]))
names(sentimentscores) <- "Score"
sentimentscores <- cbind("sentiment"=rownames(sentimentscores),sentimentscores)
rownames(sentimentscores) <- NULL
ggplot(data=sentimentscores,aes(x=sentiment,y=Score))+
  geom_bar(aes(fill=sentiment),stat = "identity")+
  theme(legend.position="none")+
  xlab("Sentiments")+ylab("Scores")+
  ggtitle("Total sentiment based on scores 2020")+
  theme_minimal()

Sentiment analysis for positive or negative tweets 2021

# Converting tweets to ASCII to trackle strange characters
tweets_sentiment_1 <- iconv(tweets, from="UTF-8", to="ASCII", sub="")

# removing retweets, in case needed 
tweets_sentiment_1 <-gsub("(RT|via)((?:\\b\\w*@\\w+)+)","",tweets_sentiment_1)

# removing mentions, in case needed
tweets_sentiment_1 <-gsub("@\\w+","",tweets_sentiment_1)
ew_sentiment<-get_nrc_sentiment((tweets_sentiment))
sentimentscores<-data.frame(colSums(ew_sentiment[,]))
names(sentimentscores) <- "Score"
sentimentscores <- cbind("sentiment"=rownames(sentimentscores),sentimentscores)
rownames(sentimentscores) <- NULL
ggplot(data=sentimentscores,aes(x=sentiment,y=Score))+
  geom_bar(aes(fill=sentiment),stat = "identity")+
  theme(legend.position="none")+
  xlab("Sentiments")+ylab("Scores")+
  ggtitle("Total sentiment based on scores 2021")+
  theme_minimal()

Visualizing the variables in the dataset 2020

#Making a ggplot
#ploting retweets by location
ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = retweet, y = location))

#ploting retweets by datetime
#need to separate date from time
ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = retweet, y = datetime), color = "blue")

Graphing users amount of followers and user amount friends

ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = user_amount_followers, y = user_amount_friends), color = "blue")

Graphing users amount status by location To check how much users Tweet by Location

ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = user_amount_status, y = location), color = "blue")

#Filter if parties are mentioned in 2020

state.name[grep ("VVD", "PvdA", "PVV", "CDA", "SP", "CU", "PvdD", state.name)]

character(0)

Visualizing the variables in the dataset 2021 ####needs amendments!!!!

#Making a ggplot
#ploting retweets by location
ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = retweet, y = location))

#ploting retweets by datetime
#need to separate date from time
ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = retweet, y = datetime), color = "blue")

Graphing users amount of followers and user amount friends

ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = user_amount_followers, y = user_amount_friends), color = "blue")

Graphing users amount status by location To check how much users Tweet by Location

ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = user_amount_status, y = location), color = "blue")

#Filter if parties are mentioned in 2021

state.name[grep ("VVD", "PvdA", "PVV", "CDA", "SP", "CU", "PvdD", state.name)]

#How can we measure how many tweets are made by location?

references: 1. https://towardsdatascience.com/a-guide-to-mining-and-analysing-tweets-with-r-2f56818fdd16 2. https://ourcodingclub.github.io/tutorials/time/ 3. https://cran.r-project.org/web/packages/stopwords/stopwords.pdf 4. https://www.dummies.com/programming/r/how-to-search-for-individual-words-in-r/ 5. https://r4ds.had.co.nz/data-visualisation.html

sum(""(BLM2020))
sum(is.na(BLM2020$Datetime))
sum(is.na(BLM2020$Text))
sum(is.na(BLM2020$Tweet.Id))
sum(is.na(BLM2020$Username))
sum(is.na(BLM2020$Location))
sum(is.na(BLM2020$Location))
summary(BLM2020)
summary(BLM2020$Location)

regression retweets

retweets_glm <-glm(Retweet ~ ., data=BLM2020)

One way ANOVA

#removing html tags from the source_tweet variable

library(textclean)
html_tags <- c(
    "<bold>Random</bold> text with symbols: &nbsp; &lt; &gt; &amp; &quot; &apos;",
    "<p>More text</p> &cent; &pound; &yen; &euro; &copy; &reg;"
)

blm2020$source_tweet<- replace_html(blm2020$source_tweet)

running the test

library(car)

one.way <- aov(source_tweet ~ ., data = blm2020)

Error in `contrasts<-`(`*tmp*`, value = contr.funs[1 + isOF[nn]]) : 
  contrasts can be applied only to factors with 2 or more levels

#Graphs for every variable #Graph on user ammount of followers #Check if a user has made multiple tweets - how can we do that? #Filter if parties are mentioned